import cv2
import numpy as np
import matplotlib.pyplot as plt
%matplotlib notebook
import utils
import os
from utils import cut # default cut function for seam finding section
sample_img_dir = 'samples/bricks_small.jpg' # feel free to change
sample_img = None
if os.path.exists(sample_img_dir):
sample_img = cv2.imread(sample_img_dir)
# plt.imshow(sample_img)
meat = cv2.imread("samples/meat.jpg")
meat = cv2.pyrDown(cv2.pyrDown(meat))[25:175, 100:250, :]
print(meat.shape)
meat = cv2.cvtColor(meat, cv2.COLOR_BGR2RGB)
drought = cv2.imread("samples/drought.jpg")[:,:733,:]
drought = cv2.pyrDown(cv2.pyrDown(drought))
cotton = cv2.imread("samples/texture.png")
text = cv2.imread("samples/text_small.jpg")
fig, ax = plt.subplots(5,1,figsize=(15,15))
ax[0].imshow(sample_img)
print(sample_img.shape)
ax[1].imshow(meat)
print(meat.shape)
ax[2].imshow(drought)
print(drought.shape)
ax[3].imshow(cotton)
print(cotton.shape)
ax[4].imshow(text)
print(text.shape)
def quilt_random(sample, out_size, patch_size):
"""
Randomly samples square patches of size patchsize from sample in order to create an output image of size outsize.
:param sample: numpy.ndarray The image you read from sample directory
:param out_size: int The width of the square output image
:param patch_size: int The width of the square sample patch
:return: numpy.ndarray
"""
# Todo
output = np.zeros([out_size, out_size, 3])
for i in range(0, out_size, patch_size):
if out_size - i < patch_size:
break
for j in range(0, out_size, patch_size):
if out_size - i < patch_size or out_size - j < patch_size:
continue
rand_x = np.random.randint(0, sample.shape[0] - patch_size)
rand_y = np.random.randint(0, sample.shape[0] - patch_size)
patch = sample[rand_x:rand_x + patch_size, rand_y : rand_y + patch_size, :]
output[i:i + patch_size, j : j + patch_size,:] = patch
return output
out_size = 500 # feel free to change to debug
patch_size = 50 # feel free to change to debug
res = quilt_random(sample_img, out_size, patch_size)
res = res.astype("int64")
plt.imshow(res)
drought_1 = quilt_random(drought, 500, 50)
plt.imshow(drought_1.astype(int))
# def ssd_patch(M, T, I):
# ## ssd
# ssd = ((M*T)**2).sum() - 2 * cv2.filter2D(I, ddepth=-1, kernel = M*T) + cv2.filter2D(I ** 2, ddepth=-1, kernel=M)
# return ssd
# def choose_sample(sample, patch_size, M, T, true_sample, tol):
# cost_image = np.zeros([sample.shape[0] - patch_size, sample.shape[0]- patch_size])
# for i in range(sample.shape[0] - patch_size):
# for j in range(sample.shape[0] - patch_size):
# cost_image[i,j] = np.sum(ssd_patch(M, T, sample[i:i+patch_size, j : j + patch_size]))
# # tenth_largest = np.sort(cost_image.ravel())[-10]
# # x,y = np.where(cost_image >= tenth_largest)
# # rand = np.random.randint(0,10)
# # x,y = (x[rand], y[rand])
# min_value = np.min(cost_image)
# row, col = np.where(cost_image <= min_value *(1+tol))
# index = np.random.randint(0, len(row))
# x,y = row[index], row[index]
# return sample[x : x + patch_size, y : y + patch_size], true_sample[x : x + patch_size, y : y + patch_size, :]
def get_all_patches(sample, patch_size):
all_patches = {}
count = -1
for i in range(sample.shape[0]):
if sample.shape[0] - i < patch_size:
break
for j in range(sample.shape[1]):
if sample.shape[1] - j < patch_size:
continue
count += 1
all_patches[count] = sample[i : i + patch_size, j : j + patch_size, :]
return all_patches
def ssd_cost(M, T, I):
useful_I = np.zeros(I.shape)
useful_I[:,:,0], useful_I[:,:,1], useful_I[:,:,2] = M * I[:,:,0], M * I[:,:,1], M * I[:,:,2]
return np.sum((T - useful_I) ** 2)
def choose_sample(sample, patch_size, M, T, tol, all_patches):
# cost_image = np.zeros([sample.shape[0] - patch_size, sample.shape[0]- patch_size])
proportion = 0.1
cost_image = {}
n = len(all_patches)
index = np.random.choice(n, int(proportion * n))
for i in index:
cost_image[i] = ssd_cost(M, T, all_patches[i])
min_value = min(cost_image.values())
choices = []
for key in cost_image.keys():
if cost_image[key] <= min_value * (1 + tol):
choices.append(all_patches[key])
return choices[np.random.choice(len(choices))]
def quilt_simple(sample, out_size, patch_size, overlap, tol):
"""
Randomly samples square patches of size patchsize from sample in order to create an output image of size outsize.
Feel free to add function parameters
:param sample: numpy.ndarray
:param out_size: int
:param patch_size: int
:param overlap: int
:param tol: int
:return: numpy.ndarray
"""
assert overlap < patch_size
# Todo
sample = sample.copy() / 255.0
patches = get_all_patches(sample, patch_size)
step = patch_size - overlap
output = np.zeros([out_size, out_size, 3])
print(len(patches))
for i in range(0, out_size + 1, step):
if out_size - i < patch_size:
break
for j in range(0, out_size + 1, step):
if out_size - j < patch_size:
continue
print(i, j)
if i == 0 and j == 0:
index = np.random.choice(len(patches))
q = index
put_new_sample(i, j, overlap, patch_size, patches[index], output)
M = (output[i : i + patch_size, j : j + patch_size, 0] != 0).astype(int)
T = (output[i : i + patch_size, j : j + patch_size])
new_sample = choose_sample(sample, patch_size, M, T, tol, patches)
# put_grey_sample(i, j, overlap, patch_size, grey_sample, grey_output)
put_new_sample(i, j, overlap, patch_size, new_sample, output)
return output
wall_2 = quilt_simple(sample_img, 500, 80, 10, 0.01)
plt.imshow(wall_2[:-20,:-20,:])
drought_2 = quilt_simple(drought , 600, 80, 30, 0.2) #feel free to change parameters to get best results
plt.figure(figsize = (4,4))
plt.imshow(drought_2[:-20, :-20, :])
def get_horizontal_mask(cost, overlap):
template = cost[:, 0 : overlap]
return cut(template.T).T
def get_vertical_mask(cost, overlap):
template = cost[0 : overlap, :]
return cut(template)
def ssd_patch(M, T, I):
## ssd
ssd = ((M*T)**2).sum() - 2 * cv2.filter2D(I, ddepth=-1, kernel = M*T) + cv2.filter2D(I ** 2, ddepth=-1, kernel=M)
return ssd
def get_all_patches(sample, patch_size):
all_patches = {}
count = -1
for i in range(sample.shape[0]):
if sample.shape[0] - i < patch_size:
break
for j in range(sample.shape[1]):
if sample.shape[1] - j < patch_size:
continue
count += 1
all_patches[count] = sample[i : i + patch_size, j : j + patch_size, :]
return all_patches
def ssd_cost(M, T, I):
# return one channel
useful_I = np.zeros(I.shape)
useful_I[:,:,0], useful_I[:,:,1], useful_I[:,:,2] = M * I[:,:,0], M * I[:,:,1], M * I[:,:,2]
return np.sum((T - useful_I) ** 2, -1)
def choose_sample_p3(sample, patch_size, M, T, tol, all_patches):
# cost_image = np.zeros([sample.shape[0] - patch_size, sample.shape[0]- patch_size])
proportion = 0.1
cost = {}
cost_image = {}
n = len(all_patches)
index = np.random.choice(n, int(proportion * n))
for i in index:
temp_cost = ssd_cost(M, T, all_patches[i])
cost[i] = np.sum(temp_cost)
cost_image[i] = temp_cost
min_value = min(cost.values())
choices = []
cost_images = []
for key in cost_image.keys():
if cost[key] <= min_value * (1 + tol):
choices.append(all_patches[key])
cost_images.append(cost_image[key])
index = np.random.choice(len(choices))
return choices[index], cost_images[index]
def fill_horizontal(i, j, patch_size, overlap, mask, rgb_output, rgb_sample):
overlap_rgb = rgb_output[i : i + patch_size, j : j + overlap, :]
for k in range(3):
overlap_rgb[:,:,k] = rgb_sample[:,:overlap,k] * mask + (1 - mask) * overlap_rgb[:,:,k]
def fill_vertical(i, j, patch_size, overlap, mask, rgb_output, rgb_sample):
overlap_rgb = rgb_output[i : i + overlap, j : j + patch_size, :]
for k in range(3):
overlap_rgb[:,:,k] = rgb_sample[:overlap,:,k] * mask + (1 - mask) * overlap_rgb[:,:,k]
# overlap_rgb = (rgb_sample[:overlap, :, :].T * rgb_mask).T + ((1 - rgb_mask) * overlap_rgb.T).T
def fill_leftover_horizontal(i, j, patch_size, overlap, rgb_output, rgb_sample):
rgb_output[i : i + patch_size, j + overlap : j + patch_size,:] = rgb_sample[:, overlap:,:]
def fill_leftover_vertical(i, j, patch_size, overlap, rgb_output, rgb_sample):
rgb_output[i + overlap : i + patch_size, j : j + patch_size,:] = rgb_sample[overlap:, :,:]
def fill_leftover(i, j, patch_size, overlap, rgb_output,rgb_sample):
rgb_output[i + overlap : i + patch_size, j + overlap : j + patch_size,:] = rgb_sample[overlap:, overlap:,:]
def put_new_sample(i, j, overlap, patch_size, new_sample, sample):
if i == 0 and j == 0:
sample[i : i + patch_size, j : j + patch_size] = new_sample
elif j == 0 and i != 0:
sample[i + overlap // 2 : i + patch_size, j : j + patch_size] = new_sample[overlap // 2:, :, :]
elif j != 0 and i == 0:
sample[i: i + patch_size, j + overlap // 2 : j + patch_size] = new_sample[:, overlap // 2:, :]
else:
sample[i + overlap // 2 : i + patch_size, j + overlap // 2 : j + patch_size] = new_sample[overlap // 2:, overlap // 2:, :]
def quilt_cut(sample, out_size, patch_size, overlap, tol):
"""
Samples square patches of size patchsize from sample using seam finding in order to create an output image of size outsize.
Feel free to add function parameters
:param sample: numpy.ndarray
:param out_size: int
:param patch_size: int
:param overlap: int
:param tol: int
:return: numpy.ndarray
"""
sample = sample.copy() / 255.0
patches = get_all_patches(sample, patch_size)
step = patch_size - overlap
output = np.zeros([out_size, out_size, 3])
for i in range(0, out_size + 1, step):
if out_size - i < patch_size:
break
for j in range(0, out_size + 1, step):
print(i, j)
if out_size - j < patch_size:
continue
M = (output[i : i + patch_size, j : j + patch_size, 0] != 0)
# print(M.shape)
T = (output[i : i + patch_size, j : j + patch_size, :])
new_sample, cost = choose_sample_p3(sample, patch_size, M, T, tol, patches)
if i == 0 and j == 0:
index = np.random.choice(len(patches))
put_new_sample(i, j, overlap, patch_size, patches[index], output)
elif i == 0 and j != 0:
mask = get_horizontal_mask(cost, overlap)
fill_horizontal(i, j, patch_size, overlap, mask, output,new_sample)
fill_leftover_horizontal(i, j, patch_size, overlap, output, new_sample)
elif i != 0 and j == 0:
mask = get_vertical_mask(cost, overlap)
fill_vertical(i, j, patch_size, overlap, mask, output, new_sample)
fill_leftover_vertical(i, j, patch_size, overlap, output, new_sample)
else:
mask = get_horizontal_mask(cost, overlap)
fill_horizontal(i, j, patch_size, overlap, mask, output, new_sample)
cost = ssd_cost(M, T, new_sample)
mask = get_vertical_mask(cost, overlap)
fill_vertical(i, j, patch_size, overlap, mask, output, new_sample)
fill_leftover(i, j, patch_size, overlap, output, new_sample)
return output
bricks = quilt_cut(sample_img, 600, 80, 20, 0.005)
plt.imshow(bricks[:-40,:-40,:])
plt.show()
meat_3 = quilt_cut(meat, 500, 80, 15, 0.01)
plt.imshow(meat_3[:-30,:-30,:])
cotton_3 = quilt_cut(cotton, 600, 80,15, 0.05)
plt.figure(figsize=(4,4))
plt.imshow(cotton_3[:-30, :-30, :])
plt.show()
drought_3 = quilt_cut(drought, 600, 80, 20, 0.3)
plt.figure(figsize=(4,4))
plt.imshow(drought_3[:-50, :-50, :])
text_3 = quilt_cut(text, 300, 50, 15, 0.1)
plt.figure(figsize = (3,3))
plt.imshow(text_3[:-10,:-10,:])
target = cv2.imread("samples/feynman.tiff")
target = cv2.pyrDown(target)
print(target.shape)
plt.imshow(target)
plt.show()
texture = cv2.imread("samples/toast.jpg")
texture = cv2.cvtColor(texture, cv2.COLOR_BGR2RGB)
texture = cv2.pyrDown(texture)[20:-20, 20:-28, :]
print(texture.shape)
plt.imshow(texture)
plt.show()
def get_all_patches(sample, patch_size):
all_patches = {}
count = -1
for i in range(sample.shape[0]):
if sample.shape[0] - i < patch_size:
break
for j in range(sample.shape[1]):
if sample.shape[1] - j < patch_size:
continue
count += 1
all_patches[count] = sample[i : i + patch_size, j : j + patch_size, :]
return all_patches
def ssd_cost(M, T, I):
# return one channel
useful_I = np.zeros(I.shape)
useful_I[:,:,0], useful_I[:,:,1], useful_I[:,:,2] = M * I[:,:,0], M * I[:,:,1], M * I[:,:,2]
return np.sum((T - useful_I) ** 2, -1)
def choose_sample_p4(patch_size, M, T, tol, target, all_patches):
# cost_image = np.zeros([sample.shape[0] - patch_size, sample.shape[0]- patch_size])
proportion = 0.05
cost = {}
cost_image = {}
n = len(all_patches)
index = np.random.choice(n, int(proportion * n))
for i in index:
alpha = np.sum(M) / (M.shape[0] * M.shape[1])
temp_cost = ssd_cost(M, T, all_patches[i])
cost2 = ssd_cost(np.ones([patch_size, patch_size]), target, all_patches[i])
cost[i] = alpha * np.sum(temp_cost) + (1 - alpha) * np.sum(cost2)
cost_image[i] = temp_cost
min_value = min(cost.values())
choices = []
cost_images = []
for key in cost_image.keys():
if cost[key] <= min_value * (1 + tol):
choices.append(all_patches[key])
cost_images.append(cost_image[key])
index = np.random.choice(len(choices))
return choices[index], cost_images[index]
def fill_horizontal(i, j, patch_size, overlap, mask, rgb_output, rgb_sample):
overlap_rgb = rgb_output[i : i + patch_size, j : j + overlap, :]
for k in range(3):
overlap_rgb[:,:,k] = rgb_sample[:,:overlap,k] * mask + (1 - mask) * overlap_rgb[:,:,k]
def fill_vertical(i, j, patch_size, overlap, mask, rgb_output, rgb_sample):
overlap_rgb = rgb_output[i : i + overlap, j : j + patch_size, :]
for k in range(3):
overlap_rgb[:,:,k] = rgb_sample[:overlap,:,k] * mask + (1 - mask) * overlap_rgb[:,:,k]
def fill_leftover_horizontal(i, j, patch_size, overlap, rgb_output, rgb_sample):
rgb_output[i : i + patch_size, j + overlap : j + patch_size,:] = rgb_sample[:, overlap:,:]
def fill_leftover_vertical(i, j, patch_size, overlap, rgb_output, rgb_sample):
rgb_output[i + overlap : i + patch_size, j : j + patch_size,:] = rgb_sample[overlap:, :,:]
def fill_leftover(i, j, patch_size, overlap, rgb_output,rgb_sample):
rgb_output[i + overlap : i + patch_size, j + overlap : j + patch_size,:] = rgb_sample[overlap:, overlap:,:]
def put_new_sample(i, j, overlap, patch_size, new_sample, sample):
if i == 0 and j == 0:
sample[i : i + patch_size, j : j + patch_size] = new_sample
elif j == 0 and i != 0:
sample[i + overlap // 2 : i + patch_size, j : j + patch_size] = new_sample[overlap // 2:, :, :]
elif j != 0 and i == 0:
sample[i: i + patch_size, j + overlap // 2 : j + patch_size] = new_sample[:, overlap // 2:, :]
else:
sample[i + overlap // 2 : i + patch_size, j + overlap // 2 : j + patch_size] = new_sample[overlap // 2:, overlap // 2:, :]
def texture_transfer(texture, target, patch_size, overlap, tol):
"""
Feel free to add function parameters
"""
texture = texture.copy() / 255.0
step = patch_size - overlap
output = target.copy() / 255.0
patches = get_all_patches(texture, patch_size)
for i in range(0, target.shape[0] + 1, step):
if target.shape[0] - i < patch_size:
break
for j in range(0, target.shape[1] + 1, step):
print(i, j)
if target.shape[1] - j < patch_size:
continue
M = (output[i : i + patch_size, j : j + patch_size, 0] != 0).astype(int)
T = (output[i : i + patch_size, j : j + patch_size, :])
rgb_texture_sample, cost = choose_sample_p4(patch_size, M, T,\
tol, output[i:i + patch_size, j : j + patch_size], patches)
if i == 0 and j == 0:
put_new_sample(i, j, overlap, patch_size, rgb_texture_sample, output)
elif i == 0 and j != 0:
mask = get_horizontal_mask(cost, overlap)
fill_horizontal(i, j, patch_size, overlap, mask, output,rgb_texture_sample)
fill_leftover_horizontal(i, j, patch_size, overlap, output, rgb_texture_sample)
elif i != 0 and j == 0:
mask = get_vertical_mask(cost, overlap)
fill_vertical(i, j, patch_size, overlap, mask, output, rgb_texture_sample)
fill_leftover_vertical(i, j, patch_size, overlap, output, rgb_texture_sample)
else:
mask = get_horizontal_mask(cost, overlap)
fill_horizontal(i, j, patch_size, overlap, mask, output, rgb_texture_sample)
cost = ssd_cost(M, T, rgb_texture_sample)
mask = get_vertical_mask(cost, overlap)
fill_vertical(i, j, patch_size, overlap, mask, output, rgb_texture_sample)
fill_leftover(i, j, patch_size, overlap, output, rgb_texture_sample)
return output
toast_man = texture_transfer(texture, target, 11, 3, 0.1)
plt.imshow(toast_man[:-10, :-10, :])
plt.imshow((cv2.GaussianBlur(toast_man[:-10,:-10,:], (3, 3), 0.5)) ** 1.05)
plt.title("gaussian blurred")
fleck = cv2.imread("samples/fleck.png")
fleck = cv2.cvtColor(fleck, cv2.COLOR_BGR2RGB)
fleck = cv2.pyrDown(fleck)
print(fleck.shape)
noodles = cv2.imread("samples/noodles.jpg")
noodles = cv2.cvtColor(noodles, cv2.COLOR_BGR2RGB)[100:-80, 50:-50, :]
noodels = cv2.pyrDown(noodles)
print(noodels.shape)
print(sky.shape)
plt.imshow(fleck)
plt.show()
plt.imshow(noodles)
noodle_fleck = texture_transfer(noodles, fleck, 10, 3, 0.05)
plt.figure(figsize=(5,3))
plt.imshow(noodle_fleck[:-20,:-20,:])
(10 pts) Create and use your own version of cut.m. To get these points, you should create your own implementation without basing it directly on the provided function (you're on the honor code for this one).
You can simply copy your customized_cut(bndcost) into the box below so that it is easier for us to grade
(15 pts) Implement the iterative texture transfer method described in the paper. Compare to the non-iterative method for two examples.
(up to 20 pts) Use a combination of texture transfer and blending to create a face-in-toast image like the one on top. To get full points, you must use some type of blending, such as feathering or Laplacian pyramid blending.
(up to 40 pts) Extend your method to fill holes of arbitrary shape for image completion. In this case, patches are drawn from other parts of the target image. For the full 40 pts, you should implement a smart priority function (e.g., similar to Criminisi et al.).
a = {"a" : 1, "b" : 2, "c" : 5, "d":3}
max(a.keys(), key = lambda x : a[x])
sorted(a.keys(), key = lambda x : a[x])